const fs = require('fs');
const Webster = require('webster');
const Consumer = Webster.consumer;

const htmlTowriter={
    html:'',
    count:0
};
class MyConsumer extends Consumer {
    constructor(option) {
        super(option);
    }
    afterCrawlRequest(result) {
        console.log('my result:', result.referInfo);
fs.writeFile('/Users/weibo/workspace/learn/webcrawler/data/'+result.referInfo.id+'.html', htmlTowriter.html, function (error) {
  if (error) {
    console.log((htmlTowriter.count++)+ ' 写入失败'+result.referInfo.id);
  } else {
    console.log((htmlTowriter.count++)+ ' 写入成功了'+result.referInfo.id);
  }
})
    }
    async beforeParseHtml(html){
        htmlTowriter.html=html;
        return true;
    }
}

let myConsumer = new MyConsumer({
    channel: 'demo_drug',
    sleepTime: 5000,
    deviceType: 'pc',
    dbConf: {
        redis: {
            host: 'redis-12419.c44.us-east-1-2.ec2.cloud.redislabs.com',
            port: 12419,
            password: 'X2AcjziaOOYPppWFOPiP4rmzZ9RFLViv'
        }
    }
});
myConsumer.startConsume();